﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using EmotionRecognitionEntities;
using EmotionRecognitionDB;
using System.Data;
using System.Windows.Forms;

namespace EmotionRecognitionManager
{
    public class Refinement
    {
        //Fc(W,Z) = E(l=1 -> k) E(i=1 -> n)[ (wli)pow(alpha) * distance(zl, xi) ]
        // Fc(W,Z) - we need to find SUCH Z , W that will minimize the Fc(W,Z) result
        // W - the belonging matrix - how much case belong to certain matrix
        // Z - the centers(centroid, maybe clusters) matrix for each cluster (emotion)
        // wli - weight of case i in cluster l
        // pow(alpha) - raise w_li in the power of alpha that will determine the variation of the W matrix
        // zl - cluster l center object (maybe certain case) composed from au's
        // xi - case i 
        //each caseid will be represented in W by his distance from the begining of the first caseid
        //same for auid 
        // Z matrix will determine the minimization of F_c(W,Z) because Z eventually determines W -> 
        /// for specific l, center of chunk l(centroid or center) : E(i=1 -> n)[distance( Zl(const) , xi)] 

        //  -> the distances between each of the center and clusters should not change too much and the process will be over
        
        //characters user dependent
        double alpha; //alpha - diff than 1
        double diffEndIter; //will decide what is  = |Fc(W(t),Z(t+1)) - Fc(W(t+1),Z(t+1))| which will end the refinement process
        //choices var
        int initCentersSelection; //will decide in which technique initial modes(centers) will be chosen most frequent(1) or, closest case which is to the frequent center in - 1(2)
        bool ispheeDist; //will decide in which techniqe to calc distance between cases and cluster centers;
        bool isHammingInt; //will choose which techniqe to compare to compare cases in case ispheeDist == false (int64 - true , bool[] false)
        double disposeCaseWeightDistPrecent; //will deside whats the range that will keep case in the system

       // bool isInitialModes;
        bool casesdisposed; //tells the process to continue to another clustering process if some cases were boundaries and removed
        //utills
        DataSet casesDs; //holds all cases of user and their au's -> will return result back to db //extract and return values of cases to db

        //data from DataBase
        //to and from caseid and index
        int[] indexCaseIdDict; //holds all casesId in system //from db [index]-caseid-
        Dictionary<int, int> caseIdIndexDict; //translate all cases to indices in CaseDict //from db <caseid,-index-> for Matrices to convert to UInt64
        int chunkAmount; //current amount of chunks for current clusetering process
		int casesAmount; //num of Casesin DS of user each of which has aus
        Dictionary<int, int> ausDict; //holds all auId of system //from db <auid,-index-> for Matrices to convert to UInt64
        //UInt64[] auMasksDict; //holds the masks to check each auIdx existence, 38 will be generated, each auIdx correspond to index in ausDict [auid-Index]-mask-

        int[] indexChunkIdDict;
        Dictionary<int, int> chunkIdIndexDict; //holds all chunkId in system //from db <chunkid,-index-> for Matrices to know which columns is chunk
        //for initial mode creation
        //double[,] freqBaseMat;

        //for refinement process
        //A -> t, B -> t+1
        //-------------------------------
        //index of arrays will represent the index - cases in casesDict
        int t, s ; // sets the time and steps vars each time new z and w, each step one of them new
        UInt64[] casesAuArr; // aus for each index - caseid, size = casesCount
        bool[][] casesAuMat;//aus columns for each index - case rows 

        int[] casesChunksIdxArr; // chunk index, for each index - caseid, size = casesCount
        int[] chunksSizesArr; //size of each chunk (number of cases) by chunk index ->chunkid
        double[,] ausFrequencyMatTrue; //matrix which holds the frequency of each au in each chunk(cluster)


        //int[,] distMatA, distMatB; //will contain all the distances from centers -  corresponds to casesDict and chunkIdIndexDict keys
        

            //for Fc(W,Z) Calculations
                //the Z(centers) matrix in both data types

        // Unsigned integer 64 bit by hamming distance only
        UInt64[] initialModesArrInt; // holds the initial centers of chunks
        UInt64[] currZArrInt;
        //boolean array by hamming/phee (new distance) measures
        bool[][] initialModesArrBool; // holds the initial centers of chunks
        bool[][] currZArrBool;
        //create centroid if chunk center is 0
        bool[] createCentroid; //for each chunk tells if the center will be from AU's frequency or from the closest Case to this frequency
            //for Fc(W,Z) Calculations
                //the W(weights) matrix
        double[,] wMatCurr;
        //for the cost functions 
        double FcA, FcB; // FcA - Fc(s-1) - prev step sum,  FcB - Fc(s) - current step sum, calculations
        double FcCurr;
        bool CostDiffA, CostDiffB;
        //for the output to db
        int[] indexCaseIdDictOutput;
        Dictionary<int, int> caseIdIndexHead; //the caseid to index dictionary which used for results return to db
        
        int[] caseIdxChunkValArrOutput; //true chunkId value for each caseIdx if -1 should dispose case
        int[] chunksSizesArrHead;
        int[] indexChunkIdDictHead;
        Dictionary<int, int> chunkIdIndexDictHead;
        UInt64[] casesAuArrHead; // aus for each index - caseid, size = casesCount
        bool[][] casesAuMatHead;
		
        public Refinement(int initCentersSelection, double alpha, double diffEndIter, bool ispheeDist, bool isHammingInt, double disposeCaseWeightDistPrecent)
        {
            this.alpha = alpha;
            this.initCentersSelection = initCentersSelection;
            this.diffEndIter = diffEndIter;
            this.ispheeDist = ispheeDist;
            this.isHammingInt = isHammingInt;
            this.disposeCaseWeightDistPrecent = disposeCaseWeightDistPrecent;
            this.casesdisposed = false;
        }
        public ReplyMsg ExecuteRefinementAlgoritm()
        {
            //get all cases of specific user and fill all relevant data structues
            BuildStructures(); //builds : ausDict chunkIdIndexDict indexChunkIdDict casesAmount indexCaseIdDict casesAuArr
            //casesChunksIdxArr casesDs casesAuMat caseIdIndexDict
            RefinementAlgoritm();
            //filter all cases which are boundary objects
            InitialOutputStructures();
            FilterCases();

            while (casesdisposed == true) //continue clustering process without boundary objects / empty clusters
            {
                ReBuildStructures();
                RefinementAlgoritm();
                FilterCases();
            }
            //filters cases by their weights
            //reassign cases to their new chunks in DataSet

            DBQueries.RefinementResultUpdate(indexCaseIdDictOutput, caseIdxChunkValArrOutput);
            return new ReplyMsg(true);

        }
        
        private void BuildStructures()
        {
            ReplyMsg ausMsg = DBQueries.GetAllAus(); // gets all aus 
            ReplyMsg chunksMsg = DBQueries.GetChunks(); // gets all list chunks
            ReplyMsg casesMsg = DBQueries.DataSetOfUserCases(); //get all cases in dataset
            ReplyMsg casesCount = DBQueries.CasesCount(); //get number of cases (with aus)
            ReplyMsg chunksCount = DBQueries.chunksCount(); //get number of chunks (with chunkid)
            //reset create centroid
            createCentroid = null;
            //handle ausDict <auid , index>
            ausDict = new Dictionary<int, int>();
            if (ausMsg.MessageState == true)
            {
                List<int> auList = (List<int>)ausMsg.ParamArray[0];//sets dict use try get value
                int i = 0;
                foreach (int au in auList)
                {
                    ausDict.Add(au, i);
                    i++;//
                }
            }
            else
                throw new Exception();

            //retrieve chunks number in system
            Dictionary<int, int> chunkSizesDict; //->for chunks dispose in rebuild
            if (chunksCount.MessageState == true)
                chunkSizesDict = (Dictionary<int, int>)chunksCount.ParamArray[0];
            else
                throw new Exception();

            //handle chunkIdIndexDict <chunkid , index> 
            chunkIdIndexDict = new Dictionary<int, int>();

            List<Chunk> chunkList;
            if (chunksMsg.MessageState == true)
            {
                chunkList = (List<Chunk>)chunksMsg.ParamArray[0];
            }
            else
                throw new Exception();
            Chunk[] chunkArr = chunkList.ToArray();
            foreach (Chunk chunkid in chunkArr)
            {
                int chunkSize = 0;
                chunkSizesDict.TryGetValue(chunkid.ChunkId, out chunkSize);
                if (chunkSize == 0)
                    chunkList.Remove(chunkid);
            }

            indexChunkIdDict = new int[chunkList.Count];
            int idx = 0;
            chunkAmount = 0;
            foreach (Chunk chunkid in chunkList)
            {
                indexChunkIdDict[idx] = chunkid.ChunkId;
                chunkIdIndexDict.Add(chunkid.ChunkId, idx);
                idx++;
                chunkAmount++;
            }
            //this steps above should remove the irrelevant chunks which are empty
            //and should not be considered in clustering process;

            //get casesamount
            if (casesCount.MessageState == true)
            {
                casesAmount = (int)casesCount.ParamArray[0];
            }
            else
                throw new Exception();

            indexCaseIdDict = new int[casesAmount]; //casesId arr by index
            casesAuArr = new UInt64[casesAmount]; //cases aus array in UInt64           
            casesChunksIdxArr = new int[casesAmount];// index chunks, for each index - caseid, size = casesCount
            //get cases data set  
            if (casesMsg.MessageState == true)
            {
                casesDs = (DataSet)casesMsg.ParamArray[0]; //gets dataset of cases , au's and their chunk
            }
            else
                throw new Exception();

            //preperation for casesDS data retrieval
            int caseidx; //which will be inserted to caseIdIndexDict and iterates indexCaseIdDict 
            casesAuMat = new bool[casesAmount][]; //holds au's of case to convert to UInt64 to casesAuArr
            for (int i = 0; i < casesAmount; i++)
            {
                casesAuMat[i] = new bool[ausDict.Count];
            }
            //indexes:        [0]     [1]   [2]   [3]
            //columns of Ds: idCase,idChunk,idAU,IsIndex
            caseIdIndexDict = new Dictionary<int, int>();
            foreach (DataTable table in casesDs.Tables) // only one table
            {
                if (table.Rows[0] != null)
                {
                    //take first line data
                    caseidx = 0; //reset case index
                    int chunkidx; //chunkidx to casesChunksIdxArr
                    if (chunkIdIndexDict.TryGetValue((int)table.Rows[caseidx][1], out chunkidx) == true)
                        casesChunksIdxArr[caseidx] = chunkidx; //reset first chunkid number
                    else throw new Exception();
                    //cases id update first
                    indexCaseIdDict[caseidx] = (int)table.Rows[caseidx][0]; //reset first caseid number
                    caseIdIndexDict.Add(indexCaseIdDict[caseidx], caseidx);
                    foreach (DataRow row in table.Rows)// each row - case with au
                    {
                        //row[0] immune to the next if
                        if (indexCaseIdDict[caseidx] != (int)row[0]) //if caseid changed assemble new line of case
                        {
                            casesAuArr[caseidx] = BoolArrToUInt64(casesAuMat[caseidx]);
                            caseidx++; //prepare for the new caseid
                            if (chunkIdIndexDict.TryGetValue((int)row[1], out chunkidx) == true)
                                casesChunksIdxArr[caseidx] = chunkidx; //reset first chunkid number
                            else throw new Exception();
                            //cases id update first
                            indexCaseIdDict[caseidx] = (int)row[0]; //reset first caseid number
                            caseIdIndexDict.Add(indexCaseIdDict[caseidx], caseidx);
                        }

                        //mark the au of this case to true
                        int auindex;
                        ausDict.TryGetValue((int)row[2], out auindex);
                        casesAuMat[caseidx][auindex] = true;
                    }
                    casesAuArr[caseidx] = BoolArrToUInt64(casesAuMat[caseidx]);
                }
            }
            ausFrequencyMatTrueCalc(); //calc frequency matrix for initial modes assign
        }

        //the algorithm:
        /* get all cases of specific user and fill all relevant data structues
         * 1. set t=1 , s=1 ,
         * 2. determine Z(t) ->set initial modes(centers) - choose according to initCentersSelection (compare final results to see which is best)
         * 3. determine W(t) according to Z(t), Fc(s) = Fc(W(t),Z(t))
         * 4. set s=s+1
         * 5. determine Z(t+1), Step A, IF : | [Fc(s-1) = Fc(W(t),Z(t))] - [Fc(s) = Fc(W(t),Z(t+1))] | <= diffEndIter, THEN : finish ELSE goto (6)
         * 6. set s = s+1
         * 7. determine W(t+1), Step B, IF : | [Fc(s-1) = Fc(W(t),Z(t+1))] - [Fc(s) = Fc(W(t+1),Z(t+1))] | <= diffEndIter, THEN finish ELSE goto (8)
         * 8. set t=t+1 , goto (4) step C
         *                          Fc(W,Z) = E(l=1 -> k) E(i=1 -> n)[ (wli)pow(alpha) * distance(zl, xi) ]
         */

        public void RefinementAlgoritm()
        {
            //step 1
            //set t=1 , s=1 ,
            CostDiffA = true;
            CostDiffB = true;
            t = 1;
            s = 1;
            //step 2
            //determine Z(t) ->set initial modes(centers) - choose according to initCentersSelection (compare final results to see which is best)
            SetInitialModes(); //Z(t) = Z(1) // uses ausFrequencyMatTrue and
            //step 3
            //determine W(t) according to Z(t), Fc(s) = Fc(W(t),Z(t))
            SetWeightsMat(); //W(t) = W(1)
           // updateStructuresStepInitial();
            InitCostFunc();
            //---------------------------------------------------------
            while (CostDiffA == true && CostDiffB == true)
            {
                t++;
                //step 4
                //set s=s+1  step A
                s++;
                PheeDistancePrepare();
                //step 5
                //determine Z(t+1) , 
                SetModes(); //Z(t+1)
                //updateStructuresStepA();
                CostFunc(); //currZArrInt/Bool & wMatCurr involved in the process
                //IF : | [Fc(s-1) = Fc(W(t),Z(t))] - [Fc(s) = Fc(W(t),Z(t+1))] | <= diffEndIter, THEN : finish ELSE goto (6)
                if (Math.Abs(FcA - FcB) <= diffEndIter) //something not reasonable same cost in both costs results
                    CostDiffA = false;
                if (CostDiffA == true)
                {
                    //----------------------------------------------------
                    //step 6
                    //set s = s+1
                    s++;
                    //step 7
                    //determine W(t+1), Step B, IF : | [Fc(s-1) = Fc(W(t),Z(t+1))] - [Fc(s) = Fc(W(t+1),Z(t+1))] | <= diffEndIter, THEN finish ELSE goto (8)
                    SetWeightsMat(); //W(t+1)
                    CostFunc();
                    //
                    if (Math.Abs(FcA - FcB) <= diffEndIter)
                        CostDiffB = false;
                    //step 8
                    t++;
                    //updateStructuresStepB();
                }
            }
        }
        
        private void FilterCases()
        {
            
            for (int i = 0; i < casesAmount; i++)
            {
                
                    double[] max2 = new double[] { 0, 0 }; //the second max weight
                    double[] max1 = new double[] { 0, 0 }; // the first max weight
                    for (int j = 0; j < chunkIdIndexDict.Count; j++)
                    {
                        if (wMatCurr[i, j] >= max1[0])
                        {
                            max2 = max1;
                            max1 = new double[] { wMatCurr[i, j], j };
                        }
                    }
                    if ((((Math.Abs(max1[0] - max2[0])) / (max1[0] + max2[0])) * 100) < disposeCaseWeightDistPrecent) //default disposeCaseWeightDistPrecent == 6.25%
                    {
                        casesdisposed = true; //for clustering loop
                        caseIdxChunkValArrOutput[caseIdIndexHead[indexCaseIdDict[i]]] = -1;
                    }
                    else
                    { //progressive chunk index to true chunk index to  true chunkid conversion
                        caseIdxChunkValArrOutput[caseIdIndexHead[indexCaseIdDict[i]]] = indexChunkIdDict[(int)max1[1]]; //from index to chunkid value  
                    }
            }

        }

        public void InitialOutputStructures()
        {
            indexCaseIdDictOutput = new int[casesAmount];
            caseIdIndexHead = new Dictionary<int, int>(); //the caseid to index dictionary which used for results return to db
            caseIdxChunkValArrOutput = new int[casesAmount]; //true chunkId value for each caseIdx if -1 should dispose case updates in filter cases
            chunksSizesArrHead = new int[chunksSizesArr.Length];
            indexChunkIdDictHead = new int[chunksSizesArr.Length];
            chunkIdIndexDictHead = new Dictionary<int, int>();
            casesAuArrHead = new UInt64[casesAmount];
            casesAuMatHead = new bool[casesAmount][];

            for (int i = 0; i < casesAmount; i++)
            {
                indexCaseIdDictOutput[i] = indexCaseIdDict[i]; // index of caseId Dictionary
            }

            {
                int i = 0;
                foreach (int caseId in caseIdIndexDict.Keys) //all caseid to index dictionary
                {
                    caseIdIndexHead.Add(caseId,i ); //copies caseid's with key i 
                    casesAuArrHead[i] = casesAuArr[i]; //moves on all cases then copies all cases au's int form
                    casesAuMatHead[i] = casesAuMat[i]; //copies also cases au in bool form
                    i++;
                }
            }
            
            for (int i = 0; i < chunksSizesArr.Length; i++)
            {
                chunksSizesArrHead[i] = chunksSizesArr[i]; //copies first chunk sizes arr
                indexChunkIdDictHead[i] = indexChunkIdDict[i]; //copies index to chunk id dict values
            }

            {
                int i = 0;
                foreach (int chunkId in chunkIdIndexDict.Keys) //copies all chunkid keys to index structure
                {
                    chunkIdIndexDictHead.Add(chunkId, i);
                    i++;
                }
            }
        }

        
        private void ReBuildStructures()///  for each new clustering process prepare following structs
        {
            casesdisposed = false;//for clustering loop
            //create new structs which removes the emptied chunks  and cases from previous cluster process
            int currCasesAmount = 0;
            chunksSizesArr = new int[chunkIdIndexDict.Count]; //calcs new chunk sizes arr - temp for data
            for (int i = 0; i < casesAmount; i++)
            { 
                if (caseIdxChunkValArrOutput[caseIdIndexHead[indexCaseIdDict[i]]] != -1) //if case not disposed - appeare in next clustering proc
                {
                    currCasesAmount++;
                    chunksSizesArr[chunkIdIndexDict[caseIdxChunkValArrOutput[caseIdIndexHead[indexCaseIdDict[i]]]]]++;
                }
            }
            chunkAmount = 0; //calcs new chunk amount
            for (int i = 0; i < chunkIdIndexDict.Count; i++)
            {
                if (chunksSizesArr[i] != 0)
                {
                    chunkAmount++;
                }
            }
            casesAmount = currCasesAmount;

            chunksSizesArr = new int[chunkAmount];
            indexChunkIdDict = new int[chunkAmount];
            chunkIdIndexDict = new Dictionary<int, int>();

            for (int i = 0, idx = 0; i < chunksSizesArrHead.Length; i++)
            {
                if (chunksSizesArrHead[i] != 0)
                {
                    chunksSizesArr[idx] = chunksSizesArrHead[i];
                    indexChunkIdDict[idx] = indexChunkIdDictHead[i];
                    chunkIdIndexDict.Add(indexChunkIdDictHead[i], idx);
                    idx++;
                }
            }
            casesAuArr = new UInt64[casesAmount];
            casesAuMat = new bool[casesAmount][];
            casesChunksIdxArr = new int[casesAmount];
            indexCaseIdDict = new int[casesAmount];
            caseIdIndexDict = new Dictionary<int,int>();
            for (int i = 0, idx = 0; i < caseIdxChunkValArrOutput.Length; i++) //iter on all starting(head) cases
            {
                if (caseIdxChunkValArrOutput[i] != -1) //caseindex to chunkid
                {
                    indexCaseIdDict[idx] = indexCaseIdDictOutput[i];
                    caseIdIndexDict.Add(indexCaseIdDictOutput[i], idx);
                    casesAuArr[idx] = casesAuArrHead[i];
                    casesAuMat[idx] = casesAuMatHead[i];
                    casesChunksIdxArr[idx] = chunkIdIndexDict[caseIdxChunkValArrOutput[i]];
                    idx++;
                }
            }
            
            
            ausFrequencyMatTrueCalc(); //calc frequency matrix for initial modes assign
        }

        private void SetWeightsMat()
        {
            wMatCurr = new double[casesAmount, chunkIdIndexDict.Count];
            int i, j;
            for (i = 0; i < casesAmount; i++)
            {

                for (j = 0; j < chunkIdIndexDict.Count; j++)
                {
                    wMatCurr[i, j] = WeightOfCaseInCluster(j, i);
                }

            }
        }

        private void InitCostFunc()
        {
            FcCurr = 0;
            //FcA
            for (int i = 0; i < chunkIdIndexDict.Count; i++)
            {
                for (int j = 0; j < casesAmount; j++)
                {
                    FcCurr += Math.Pow(wMatCurr[j, i], alpha) * DistanceMetricWCluster(i, j);
                }
            }
            FcA = FcCurr;
            FcB = FcCurr;
        }

        private void CostFunc()
        {
            FcCurr=0;
            //FcA
            for (int i = 0; i < chunkIdIndexDict.Count; i++)
            {
                for (int j = 0; j < casesAmount; j++)
                {
                    
                        FcCurr += Math.Pow(wMatCurr[j, i], alpha) * DistanceMetricWCluster(i, j);

                }
            }

            FcA = FcB;
            FcB = FcCurr;
        }


        private void PheeDistancePrepare()
        {
            if (ispheeDist == true)
            {
                ReassignCasesToChunks(); //reaasign cases to chunks for phee distance calculations
                ausFrequencyMatTrueCalc(); //calc frequency matrix for phee distance calculations
            }
        }
        //the modes table creation, not by initial function!
        private void SetModes()
        {
            bool isDistinctModes = false;
            bool[][] currZArrBool = new bool[chunkIdIndexDict.Count][]; //holds au's of case to convert to UInt64 to casesAuArr
            for (int i = 0; i < chunkIdIndexDict.Count; i++)
            {
                currZArrBool[i] = new bool[ausDict.Count];
            }

            for (int chunkIdx = 0; chunkIdx < chunkIdIndexDict.Count; chunkIdx++)
            {
                for (int AuIdx = 0; AuIdx < ausDict.Count; AuIdx++)
                {
                    double sumTrue = 0, sumFalse = 0;
                    for (int caseIdx = 0; caseIdx < casesAmount; caseIdx++)
                    {

                        if (casesAuMat[caseIdx][AuIdx] == true)
                            sumTrue += wMatCurr[caseIdx, chunkIdx];
                        else
                            sumFalse += wMatCurr[caseIdx, chunkIdx];

                    }
                    if (sumTrue >= sumFalse) //calibrate here >= to = if necessary
                        currZArrBool[chunkIdx][AuIdx] = true;
                }
                currZArrInt[chunkIdx] = BoolArrToUInt64(currZArrBool[chunkIdx]);
            }
            createCentroid = new bool[chunkIdIndexDict.Count];
            //check if mode is unique if not the clustering process will not converge
            for (int chunkI = 0; chunkI < chunkIdIndexDict.Count; chunkI++)
            {
                for (int chunkJ = 0; chunkJ < chunkIdIndexDict.Count; chunkJ++)
                {
                    if ((currZArrInt[chunkI] == currZArrInt[chunkJ]) && (chunkI != chunkJ) && (currZArrInt[chunkI] != 0))
                    {
                        createCentroid[chunkI] = true;
                        isDistinctModes = true;
                    }
                }
            }
            if(isDistinctModes == true)
                DistinctModes();
        }
        private void DistinctModes()
        {
            //if not then only zero centers will have aus of minimum centroids of that cluster
            double[,] AuWeightsDiff = new double[chunkIdIndexDict.Count, ausDict.Count];
            bool[][] CenterModesArrBool = new bool[chunkIdIndexDict.Count][];
            UInt64[] CenterModesArrInt = new UInt64[chunkIdIndexDict.Count];
            double[] AuWeightsTake = new double[chunkIdIndexDict.Count];

            for (int chunkIdx = 0; chunkIdx < chunkIdIndexDict.Count; chunkIdx++)
            {
                for (int AuIdx = 0; AuIdx < ausDict.Count; AuIdx++)
                {
                    double sumTrue = 0, sumFalse = 0;
                    for (int caseIdx = 0; caseIdx < casesAmount; caseIdx++)
                    {

                        if (casesAuMat[caseIdx][AuIdx] == true)
                            sumTrue += wMatCurr[caseIdx, chunkIdx];
                        else
                            sumFalse += wMatCurr[caseIdx, chunkIdx];

                    }
                    AuWeightsDiff[chunkIdx, AuIdx] = sumTrue - sumFalse; //look for smaller than AuWeightsDiff value
                }
            }
            //prepare temp initial centers which correspond to cases index; if center is 0 -> the most minimum centroid will be chosen
            int[] tempInitialModesArr = new int[chunkIdIndexDict.Count];
            for (int k = 0; k < chunkIdIndexDict.Count; k++)
            {

                if (initialModesArrInt[k] != 0) // find au's for not unique centers
                {
                    double tempFreq = 100000;
                    //initial Au frequency to compare to
                    for (int j = 0; j < ausDict.Count; j++)
                    {
                        if (currZArrBool[k][j] == true)
                            tempFreq = AuWeightsDiff[k, j];
                    }
                    //find the lowest Au frequency in the current center
                    for (int j = 0; j < ausDict.Count; j++)
                    {
                        if (currZArrBool[k][j] == true) //au in current center
                        {
                            if (AuWeightsDiff[k, j] < tempFreq) //if there is lower AUweightdiff than the curr in curr center than replace older
                            {
                                tempFreq = AuWeightsDiff[k, j];
                            }
                        }
                    }
                    //find the next au's which their frequency is the next lowest to prevent ununiqueness
                    double HighestUnderTempFreq = 0;
                    for (int j = 0; j < ausDict.Count; j++)
                    {
                        if (AuWeightsDiff[k, j] < tempFreq)
                        {
                            HighestUnderTempFreq = AuWeightsDiff[k, j];
                        }
                    }
                    for (int j = 0; j < ausDict.Count; j++)
                    {
                        if ((HighestUnderTempFreq < AuWeightsDiff[k, j]) && (AuWeightsDiff[k, j] < tempFreq))
                        {
                            HighestUnderTempFreq = AuWeightsDiff[k, j];
                        }
                    }
                    AuWeightsTake[k] = HighestUnderTempFreq;
                }
                
                //get the au's for each center which their frequency is equal to modesFreq[centerId]
                CenterModesArrBool[k] = new bool[ausDict.Count]; //create temp bool[] to contain aus of new center
                for (int j = 0; j < ausDict.Count; j++)
                {
                    if (AuWeightsDiff[k, j] == AuWeightsTake[k]) // if frequency is same as we look for than take au for center
                        CenterModesArrBool[k][j] = true;
                }
                CenterModesArrInt[k] = BoolArrToUInt64(CenterModesArrBool[k]);
            }
            //add aus to centers which are zeros or not unique to the calculated center based minimal centroids
            for (int i = 0; i < chunkIdIndexDict.Count; i++) //malfunction error correct 
            {
                if (createCentroid[i] == true)
                {
                    for (int j = 0; j < ausDict.Count; j++)
                    {
                        if (CenterModesArrBool[i][j] == true)
                        {
                            currZArrBool[i][j] = true;
                        }
                    }

                    currZArrInt[i] = BoolArrToUInt64(currZArrBool[i]);
                }
            }

            //check if modes are distinct and unique if not call this func again
            bool callDistinctModes = false;
            for (int chunkI = 0; chunkI < chunkIdIndexDict.Count; chunkI++)
            {
                for (int chunkJ = 0; chunkJ < chunkIdIndexDict.Count; chunkJ++)
                {
                    if ((currZArrInt[chunkI] == currZArrInt[chunkJ]) && (chunkI != chunkJ) && (currZArrInt[chunkI] != 0))
                    {
                        createCentroid[chunkI] = true;
                        callDistinctModes = true;
                    }
                }
            }
            if (callDistinctModes == true)
                DistinctModes();
        }
        private void ausFrequencyMatTrueCalc()
        {
            chunksSizesArr = new int[chunkIdIndexDict.Count];
            //count number of cases per chunk
            //count cases in chunks
            for (int i = 0; i < casesAmount; i++)
            {
                chunksSizesArr[casesChunksIdxArr[i]]++;
            }
            //frequency matrix reset
            ausFrequencyMatTrue = new double[chunkIdIndexDict.Count, ausDict.Count];
            //check frequency appearences of au's
            //matrix of chunks to aus for the frequency of auI in clusterJ
            for (int i = 0; i < casesChunksIdxArr.Length; i++)
            {
                for (int j = 0; j < ausDict.Count; j++)
                {
                    if (casesAuMat[i][j] == true)
                        ausFrequencyMatTrue[casesChunksIdxArr[i], j] += (1 / (double)(chunksSizesArr[casesChunksIdxArr[i]]));
                }
            }
        }

        //if auj appears in modex then 
        //auj apearence in all cases of this mode / the sum of cases ->
        //-> is the largest among all cases
        private void SetInitialModes() //create situation where chunks have no centers because the count of cases in them and frequency of aus is low
        {
          //  isInitialModes = true;
            bool isNotUnique = false;
            bool zeromodes = false;
            initialModesArrInt = new UInt64[chunkIdIndexDict.Count];
            initialModesArrBool = new bool[chunkIdIndexDict.Count][];
            for (int k = 0; k < chunkIdIndexDict.Count; k++)
            {
                initialModesArrBool[k] = new bool[ausDict.Count]; //create temp bool[] to contain aus of center
                for (int j = 0; j < ausDict.Count; j++)
                {
                    if (ausFrequencyMatTrue[k, j] >= 0.5) // if frequency >= 0.5 store au in center
                        initialModesArrBool[k][j] = true;
                }
                initialModesArrInt[k] = BoolArrToUInt64(initialModesArrBool[k]);
            }
            //createCentroid[chunks] - boolean array for centroid cluster creation
            //check if modes exists
            //create situation where chunks have no centers because the count of cases in them and frequency of aus is low
            createCentroid = new bool[chunkIdIndexDict.Count];
            for (int k = 0; k < chunkIdIndexDict.Count; k++)
            {
                if (initialModesArrInt[k] == (UInt64)0)
                {
                    createCentroid[k] = true;
                    zeromodes = true;
                }
            }
            //check if modes are unique
            for (int chunkI = 0; chunkI < chunkIdIndexDict.Count; chunkI++)
            {
                for (int chunkJ = 0; chunkJ < chunkIdIndexDict.Count; chunkJ++)
                {
                    if ((initialModesArrInt[chunkI] == initialModesArrInt[chunkJ]) && (chunkI != chunkJ) && (initialModesArrInt[chunkI] != 0))
                    {
                        createCentroid[chunkI] = true;
                        isNotUnique = true;
                    }
                }
            }
            currZArrInt = initialModesArrInt; //sets current centers array
            currZArrBool = initialModesArrBool;
            if ((isNotUnique == true) || (zeromodes == true))
                InitDistinctModes();
        }

        //create init modes which are closest to clusters centers claced before 
        private void InitDistinctModes()
        {
            //if not then only zero centers will have aus of minimum centroids of that cluster
            double[] modesFreq = new double[chunkIdIndexDict.Count];
            bool[][] initialCentroidModesArrBool = new bool[chunkIdIndexDict.Count][];
            UInt64[] initialCentroidModesArrInt = new UInt64[chunkIdIndexDict.Count];

            for (int k = 0; k < chunkIdIndexDict.Count; k++)
            {
                
                if (initialModesArrInt[k] != 0) // find au's for not unique centers
                {
                    double tempFreq = 100000;
                    //initial Au frequency to compare to
                    for (int j = 0; j < ausDict.Count; j++)
                    {
                        if (initialModesArrBool[k][j] == true)
                            tempFreq = ausFrequencyMatTrue[k, j];
                    }
                    //find the lowest Au frequency in the current center
                    for (int j = 0; j < ausDict.Count; j++)
                    {
                        if (initialModesArrBool[k][j] == true) //au in current center
                        {
                            if (ausFrequencyMatTrue[k, j] < tempFreq) //if there is lower AUFreq than the curr in curr center than replace older
                            {
                                tempFreq = ausFrequencyMatTrue[k, j];
                            }
                        }
                    }
                    //find the next au's which their frequency is the next lowest to prevent ununiqueness
                    double HighestUnderTempFreq = 0;
                    for (int j = 0; j < ausDict.Count; j++)
                    {
                        if (ausFrequencyMatTrue[k, j] < tempFreq)
                        {
                            HighestUnderTempFreq = ausFrequencyMatTrue[k, j];
                        }
                    }
                    for (int j = 0; j < ausDict.Count; j++)
                    {
                        if ( (HighestUnderTempFreq < ausFrequencyMatTrue[k, j]) && (ausFrequencyMatTrue[k, j] < tempFreq) )
                        {
                            HighestUnderTempFreq = ausFrequencyMatTrue[k, j];
                        }
                    }
                    modesFreq[k] = HighestUnderTempFreq;
                }
                else//find highest frequecy aus value for the zero centers
                {
                    double tempFreq = 0;
                    for (int j = 0; j < ausDict.Count; j++)
                    {
                        if (ausFrequencyMatTrue[k, j] > tempFreq)
                        {
                            tempFreq = ausFrequencyMatTrue[k, j];
                        }
                    }
                    modesFreq[k] = tempFreq;
                }
                //get the au's for each center which their frequency is equal to modesFreq[centerId]
                initialCentroidModesArrBool[k] = new bool[ausDict.Count]; //create temp bool[] to contain aus of new center
                for (int j = 0; j < ausDict.Count; j++)
                {
                    if (ausFrequencyMatTrue[k, j] == modesFreq[k]) // if frequency is same as we look for than take au for center
                        initialCentroidModesArrBool[k][j] = true;
                }
                initialCentroidModesArrInt[k] = BoolArrToUInt64(initialCentroidModesArrBool[k]);
            }
           
            //add aus to centers which are zeros or not unique to the calculated center based minimal centroids
            for (int i = 0; i < chunkIdIndexDict.Count; i++) 
            {
                if (createCentroid[i] == true)
                {
                    for (int j = 0; j < ausDict.Count; j++)
                    {
                        if (initialCentroidModesArrBool[i][j] == true)
                        {
                            currZArrBool[i][j] = true;
                            initialModesArrBool[i][j] = true;
                        }
                    }

                    currZArrInt[i] = BoolArrToUInt64(currZArrBool[i]);
                    initialModesArrInt[i] = BoolArrToUInt64(initialModesArrBool[i]);
                }
            }
            //check if modes are distinct and unique if not call this func again
            bool callDistinctModes = false;
            for (int chunkI = 0; chunkI < chunkIdIndexDict.Count; chunkI++)
            {
                for (int chunkJ = 0; chunkJ < chunkIdIndexDict.Count; chunkJ++)
                {
                    if ((currZArrInt[chunkI] == currZArrInt[chunkJ]) && (chunkI != chunkJ) && (currZArrInt[chunkI] != 0) )
                    {
                        createCentroid[chunkI] = true;
                        callDistinctModes = true;
                    }
                }
            }
            if (callDistinctModes == true)
                InitDistinctModes();
        }
        
        //wli = {1, xi=Zl ; 0, xi=zh 
        // ; 1/{ {sum(1->|chunks-1|) [(distance(zl, xi)/distance(zh, xi)]} ^ 1/(alpha-1)}, (xi != zl)&&(xi != zh), 1<=h<=k, alpha -> 1^+} // alpha is out of sum
        private double WeightOfCaseInCluster(int clusterIdx, int caseIdx)
        {
            double wli;
            double[] distanceClusterWCaseIdx = new double[chunkIdIndexDict.Count];
            for (int i = 0; i < chunkIdIndexDict.Count; i++)
            {
                distanceClusterWCaseIdx[i] = DistanceMetricWCluster(i, caseIdx); //for later calcs
                if (distanceClusterWCaseIdx[i] == 0)
                {
                    if (i == clusterIdx)
                        return 1;
                    else
                        return 0;
                }

            }
            //if case not equal to each of centers
            //calculate the weight;
            double innerSum = 0;
            double dZlXi = distanceClusterWCaseIdx[clusterIdx];
            for (int i = 0; i < chunkIdIndexDict.Count; i++)
            {
                innerSum = innerSum + (Math.Pow((dZlXi / distanceClusterWCaseIdx[i]), (1 / (alpha - 1))));
            }
            wli = 1 / innerSum;
            return wli;
        }

        private void ReassignCasesToChunks()
        {
            casesChunksIdxArr = new int[casesAmount];
            for (int i = 0; i < casesAmount; i++)
            {
                double[] max1 = new double[] { 0, 0 }; // the first max weight
                for (int j = 0; j < chunkIdIndexDict.Count; j++)//arbitarily last max weight chunk is selected for the case
                {
                    
                    if (wMatCurr[i, j] == max1[0] && (chunksSizesArr[j] > chunksSizesArr[(int)max1[1]]))
                    {
                        max1 = new double[] { wMatCurr[i, j], j };
                    }
                    else if (wMatCurr[i, j] > max1[0])
                    {
                        max1 = new double[] { wMatCurr[i, j], j };
                    }
                }
                casesChunksIdxArr[i] = (int)max1[1]; //take care of equal breaker (random or other kind)
            }
        }






        //the measures choosing
        public int DistanceCasesHammingIdx(int caseIdxA, int caseIdxB) //indices choose method by isHammingInt hamming UInt64 ot bool[]
        {
            switch (isHammingInt)
            {
                case true:
                    return DistanceHammingInt(casesAuArr[caseIdxA], casesAuArr[caseIdxB]);
                case false:
                    return DistanceHammingBool(casesAuMat[caseIdxA], (casesAuMat[caseIdxB]));
            }
            return -1;
        }
        public int DistanceCasesHammingValue(object caseA, object caseB) //not usable - objects choose method by isHammingInt hamming UInt64 ot bool[] according to objects sent
        {
                if(caseA is UInt64)
                    return DistanceHammingInt((UInt64)caseA, (UInt64)caseB);
                if (caseA is bool[])
                    return DistanceHammingBool((bool[])caseA, (bool[])caseB);
            return -1;
        }
        //used in DistanceMetricWCluster:

        //distance measures
        //primary
        public double DistanceMetricWCluster(int centerIdx, int caseIdx) //indices choose method by ispheeDist hamming or phee
        {
            switch (ispheeDist)
            {
                case false:
                    return DistanceWCenterHammingIdx(centerIdx, caseIdx); //DistanceHammingBool((bool[])a, (bool[])b)
                case true:
                    if (isHammingInt == false)
                        return DistancePhee(centerIdx, caseIdx);
                    else
                        throw new Exception();
            }
            return -1;
        }
        //secondary when choosing this 
        public int DistanceWCenterHammingIdx(int centerIdx, int caseIdx) //indices choose method by isHammingInt hamming UInt64 ot bool[]
        {
            switch (isHammingInt)
            {
                case true:
                    return DistanceHammingInt(currZArrInt[centerIdx], casesAuArr[caseIdx]);
                case false:
                    return DistanceHammingBool(currZArrBool[centerIdx], casesAuMat[caseIdx]);
            }
            return -1;
        }

        //the measures
         //only between cluster index and case au's & only when using DistanceHammingBool
        public double DistancePhee(int centerIdx, int caseIdx)//new metric distance between cases and center
        {
            double sum = 0;
            for (int i = 0; i < ausDict.Count; i++)
            {
                if (currZArrBool[centerIdx][i] == casesAuMat[caseIdx][i])
                {
                    if (casesAuMat[caseIdx][i] == true)//if(casesAuMat[caseIdx][i] == true)
                        sum += (1 - ausFrequencyMatTrue[centerIdx, i]); // represents the true valuse of attribute value of AUi
                   // else
                        //sum += ausFrequencyMatTrue[centerIdx, i];
                } 
                else
                    sum += 1;
            }
            return sum;
        }


        public int DistanceHammingInt(UInt64 centerIdx, UInt64 caseIdx) //hamming distance between 64 bit integers(UInt64) ,38 aus -> 26 bits unused
        {
            int distance = 0;
            UInt64 xorRes = centerIdx ^ caseIdx;
            while (xorRes > 0)
            {
                distance++;
                xorRes = xorRes & (xorRes - 1);
            }
            return distance;
        }

        public int DistanceHammingBool(bool[] centerIdx, bool[] caseIdx) //hamming distance between booleans arrays
        {
            int distance = 0;
            int idxB, idxA;
            //for (idx = aus.Count - 1; idx >= 0 ; idx--) //for true run
            int maxlength = Math.Max(centerIdx.Length, caseIdx.Length);
            for (idxA = centerIdx.Length - 1, idxB = caseIdx.Length - 1; Math.Max(idxA, idxB) >= 0; idxB--, idxA--) //test only
            {
                try
                {
                    if (centerIdx[idxA] != caseIdx[idxB])
                    {
                        distance++;
                    }
                }
                catch (IndexOutOfRangeException)
                {
                    if (Math.Max(centerIdx.Length, caseIdx.Length) == centerIdx.Length)
                    {
                        if (centerIdx[idxA] == true)
                            distance++;
                    }
                    else
                    {
                        if (centerIdx[idxA] == true)
                            distance++;
                    }
                }
            }
            return distance;
        }


        public UInt64 Mask(int auIdx) //create mask for au extraction in 64 bit var
        {//if( ( ((UInt64)num ->to check if auidx exist) & (UInt64)Mask(auidx) ) != 0) ? au exists : au not exists
            string maskStr = "";
            int i;
            for (i = 0; i < auIdx; i++)
                maskStr = maskStr + "0";
            maskStr = "1" + maskStr; // fill index to 1
            for (i = 0; i < 64 - auIdx; i++)
                maskStr = "0" + maskStr;
            return Convert.ToUInt64(maskStr, 2);
        }

        ////STEP Init - W(curr) = W(1) = W(t), Z(curr) = Z(1) = Z(t)
        //private void updateStructuresStepInitial()
        //{
        //    wMatA = wMatCurr;
        //    wMatB = wMatCurr;
        //    if (isHammingInt == true)
        //    {
        //        zArrAInt = currZArrInt;
        //        zArrBInt = currZArrInt;
        //    }
        //    else
        //    {
        //        zArrABool = currZArrBool;
        //        zArrBBool = currZArrBool;
        //    }

        //}
        ////STEP A - Z(t+1) = Z(curr), t - change -> replace structures
        //private void updateStructuresStepA()
        //{
        //    if (isHammingInt == true)
        //    {

        //        zArrAInt = zArrBInt;
        //        zArrBInt = currZArrInt;
        //    }
        //    else
        //    {
        //        zArrABool = zArrBBool;
        //        zArrBBool = currZArrBool;
        //    }

        //}
        ////STEP B - W(t+1) = W(curr), t - change -> replace structures
        //private void updateStructuresStepB()
        //{
        //    wMatA = wMatB;
        //    wMatB = wMatCurr;
        //}

        #region ConvertUtills for refinement

        public static string ConvertToBinaryString(UInt64 value) //converts 64 bit integer to string
        {
            if (value == 0) return "0";
            System.Text.StringBuilder b = new System.Text.StringBuilder();
            while (value != 0)
            {
                b.Insert(0, ((value & 1) == 1) ? '1' : '0');
                value >>= 1;
            }
            return string.Format(b.ToString().PadLeft(68, '0'));
        }
        public static UInt64 BoolArrToUInt64(bool[] a) //converts the boolean array to 64 bit integer 
        {
            return Convert.ToUInt64(BoolArrToString(a), 2);
        }
        public static string BoolArrToString(bool[] a)//converts boolean array to binarical number string 
        {
            string aStr = "";
            foreach (bool val in a)
            {
                aStr = ((val == true) ? "1" : "0") + aStr;
            }
            return aStr;
        }
        public static int ResultDistance(UInt64 xorRes) //counts number of ones
        {
            int distance = 0;
            while (xorRes > 0)
            {
                distance++;
                xorRes = xorRes & (xorRes - 1);
            }
            return distance;
        }

        #endregion //all the conversion utills

        //public void RefinementTests()
        //{
        //    //testing utilities
        //    UInt64 a1 = 3; // 0...0011

        //    UInt64 b1 = 5; // 0...0101
        //    //          xor  0...0110
        //    bool[] a2 = { false, false, true, true };
        //    bool[] b2 = { true, false, true };


        //    MessageBox.Show(string.Format("1.(UInt64){0} = (UInt64->string){1}", a1, ConvertToBinaryString(a1)));
        //    MessageBox.Show(string.Format("2.(UInt64){0} = (UInt64->string){1}", b1, ConvertToBinaryString(b1)));
        //    MessageBox.Show(string.Format("3.(UInt64){0} = (UInt64->string){1}", b1 ^ a1, ConvertToBinaryString(a1 ^ b1)));

        //    MessageBox.Show(string.Format("4.distance (UInt64,UInt64) -> (int){0}", Distance(a1, b1)));

        //    MessageBox.Show(string.Format("5.distance (bool[],bool[]) -> (int){0}", Distance(a2, b2)));

        //    MessageBox.Show(string.Format("6.(bool[]->string){0} = (bool[]->UInt64){1}", BoolArrToString(a2), BoolArrToUInt64(a2)));
        //    MessageBox.Show(string.Format("7.(bool[]->string){0} = (bool[]->UInt64){1}", BoolArrToString(b2), BoolArrToUInt64(b2)));
        //    MessageBox.Show(string.Format("8.distance -> [((bool[]->UInt64)^(bool[]->UInt64))->string]{0}", ConvertToBinaryString(BoolArrToUInt64(a2) ^ BoolArrToUInt64(b2))));

        //}
   
    }//
}